{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Week 5 Day 2\n",
    "\n",
    "### AutoGen AgentChat - Going deeper..\n",
    "\n",
    "1. Multi-modal conversation\n",
    "2. Structured Outputs\n",
    "3. Using LangChain tools\n",
    "4. Teams\n",
    "\n",
    "...and a special surprise extra piece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import BytesIO\n",
    "import requests\n",
    "from autogen_agentchat.messages import TextMessage, MultiModalMessage\n",
    "from autogen_core import Image as AGImage\n",
    "from PIL import Image\n",
    "from dotenv import load_dotenv\n",
    "from autogen_ext.models.openai import OpenAIChatCompletionClient\n",
    "from autogen_agentchat.agents import AssistantAgent\n",
    "from autogen_core import CancellationToken\n",
    "from IPython.display import display, Markdown\n",
    "from pydantic import BaseModel, Field\n",
    "from typing import Literal\n",
    "\n",
    "load_dotenv(override=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A multi-modal conversation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://edwarddonner.com/wp-content/uploads/2024/10/from-software-engineer-to-AI-DS.jpeg\"\n",
    "\n",
    "pil_image = Image.open(BytesIO(requests.get(url).content))\n",
    "img = AGImage(pil_image)\n",
    "img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_modal_message = MultiModalMessage(content=[\"Describe the content of this image in detail\", img], source=\"User\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_client = OpenAIChatCompletionClient(model=\"gpt-4o-mini\")\n",
    "\n",
    "describer = AssistantAgent(\n",
    "    name=\"description_agent\",\n",
    "    model_client=model_client,\n",
    "    system_message=\"You are good at describing images\",\n",
    ")\n",
    "\n",
    "response = await describer.on_messages([multi_modal_message], cancellation_token=CancellationToken())\n",
    "reply = response.chat_message.content\n",
    "display(Markdown(reply))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Structured Outputs!\n",
    "\n",
    "Autogen AgentChat makes it easy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class ImageDescription(BaseModel):\n",
    "    scene: str = Field(description=\"Briefly, the overall scene of the image\")\n",
    "    message: str = Field(description=\"The point that the image is trying to convey\")\n",
    "    style: str = Field(description=\"The artistic style of the image\")\n",
    "    orientation: Literal[\"portrait\", \"landscape\", \"square\"] = Field(description=\"The orientation of the image\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_client = OpenAIChatCompletionClient(model=\"gpt-4o-mini\")\n",
    "\n",
    "describer = AssistantAgent(\n",
    "    name=\"description_agent\",\n",
    "    model_client=model_client,\n",
    "    system_message=\"You are good at describing images in detail\",\n",
    "    output_content_type=ImageDescription,\n",
    ")\n",
    "\n",
    "response = await describer.on_messages([multi_modal_message], cancellation_token=CancellationToken())\n",
    "reply = response.chat_message.content\n",
    "reply"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import textwrap\n",
    "print(f\"Scene:\\n{textwrap.fill(reply.scene)}\\n\\n\")\n",
    "print(f\"Message:\\n{textwrap.fill(reply.message)}\\n\\n\")\n",
    "print(f\"Style:\\n{textwrap.fill(reply.style)}\\n\\n\")\n",
    "print(f\"Orientation:\\n{textwrap.fill(reply.orientation)}\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using LangChain tools from AutoGen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AutoGen's wrapper:\n",
    "\n",
    "from autogen_ext.tools.langchain import LangChainToolAdapter\n",
    "\n",
    "# LangChain tools:\n",
    "\n",
    "from langchain_community.utilities import GoogleSerperAPIWrapper\n",
    "from langchain_community.agent_toolkits import FileManagementToolkit\n",
    "from langchain.agents import Tool\n",
    "\n",
    "\n",
    "prompt = \"\"\"Your task is to find a one-way non-stop flight from JFK to LHR in June 2025.\n",
    "First search online for promising deals.\n",
    "Next, write all the deals to a file called flights.md with full details.\n",
    "Finally, select the one you think is best and reply with a short summary.\n",
    "Reply with the selected flight only, and only after you have written the details to the file.\"\"\"\n",
    "\n",
    "\n",
    "serper = GoogleSerperAPIWrapper()\n",
    "langchain_serper =Tool(name=\"internet_search\", func=serper.run, description=\"useful for when you need to search the internet\")\n",
    "autogen_serper = LangChainToolAdapter(langchain_serper)\n",
    "autogen_tools = [autogen_serper]\n",
    "\n",
    "langchain_file_management_tools = FileManagementToolkit(root_dir=\"sandbox\").get_tools()\n",
    "for tool in langchain_file_management_tools:\n",
    "    autogen_tools.append(LangChainToolAdapter(tool))\n",
    "\n",
    "for tool in autogen_tools:\n",
    "    print(tool.name, tool.description)\n",
    "\n",
    "model_client = OpenAIChatCompletionClient(model=\"gpt-4o-mini\")\n",
    "agent = AssistantAgent(name=\"searcher\", model_client=model_client, tools=autogen_tools, reflect_on_tool_use=True)\n",
    "message = TextMessage(content=prompt, source=\"user\")\n",
    "result = await agent.on_messages([message], cancellation_token=CancellationToken())\n",
    "for message in result.inner_messages:\n",
    "    print(message.content)\n",
    "display(Markdown(result.chat_message.content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we need to call the agent again to write the file\n",
    "\n",
    "message = TextMessage(content=\"OK proceed\", source=\"user\")\n",
    "\n",
    "result = await agent.on_messages([message], cancellation_token=CancellationToken())\n",
    "for message in result.inner_messages:\n",
    "    print(message.content)\n",
    "display(Markdown(result.chat_message.content))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Team interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from autogen_agentchat.agents import AssistantAgent\n",
    "from autogen_agentchat.conditions import  TextMentionTermination\n",
    "from autogen_agentchat.teams import RoundRobinGroupChat\n",
    "\n",
    "from autogen_ext.tools.langchain import LangChainToolAdapter\n",
    "from langchain_community.utilities import GoogleSerperAPIWrapper\n",
    "from langchain.agents import Tool\n",
    "\n",
    "serper = GoogleSerperAPIWrapper()\n",
    "langchain_serper =Tool(name=\"internet_search\", func=serper.run, description=\"useful for when you need to search the internet\")\n",
    "autogen_serper = LangChainToolAdapter(langchain_serper)\n",
    "\n",
    "model_client = OpenAIChatCompletionClient(model=\"gpt-4o-mini\")\n",
    "\n",
    "\n",
    "prompt = \"\"\"Find a one-way non-stop flight from JFK to LHR in June 2025.\"\"\"\n",
    "\n",
    "\n",
    "primary_agent = AssistantAgent(\n",
    "    \"primary\",\n",
    "    model_client=model_client,\n",
    "    tools=[autogen_serper],\n",
    "    system_message=\"You are a helpful AI research assistant who looks for promising deals on flights. Incorporate any feedback you receive.\",\n",
    ")\n",
    "\n",
    "evaluation_agent = AssistantAgent(\n",
    "    \"evaluator\",\n",
    "    model_client=model_client,\n",
    "    system_message=\"Provide constructive feedback. Respond with 'APPROVE' when your feedback is addressed.\",\n",
    ")\n",
    "\n",
    "text_termination = TextMentionTermination(\"APPROVE\")\n",
    "\n",
    "# With thanks to Peter A for adding in the max_turns - otherwise this can get into a loop..\n",
    "\n",
    "team = RoundRobinGroupChat([primary_agent, evaluation_agent], termination_condition=text_termination, max_turns=20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = await team.run(task=prompt)\n",
    "for message in result.messages:\n",
    "    print(f\"{message.source}:\\n{message.content}\\n\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drumroll..\n",
    "\n",
    "## Introducing MCP!\n",
    "\n",
    "Our first look at the Model Context Protocol from Anthropic -\n",
    "\n",
    "Autogen makes it easy to use MCP tools, just like LangChain tools."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table style=\"margin: 0; text-align: left; width:100%\">\n",
    "    <tr>\n",
    "        <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
    "            <img src=\"../assets/stop.png\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
    "        </td>\n",
    "        <td>\n",
    "            <h2 style=\"color:#ff7800;\">But wait - a not-so-small problem for Windows PC people</h2>\n",
    "            <span style=\"color:#ff7800;\">I have unpleasant news. There's a problem running MCP Servers on Windows PCs; Mac and Linux is fine. This is a known issue as of May 4th, 2025. I asked o3 with Deep Research to try to find workarounds; it <a href=\"https://chatgpt.com/share/6817bbc3-3d0c-8012-9b51-631842470628\">confirmed the issue</a> and confirmed the workaround.<br/><br/>\n",
    "            The workaround is a bit of a bore. It is to take advantage of \"WSL\", the Microsoft approach for running Linux on your PC. You'll need to carry out more setup instructions! But it's quick, and several students have confirmed that this works perfectly for them, then this lab and the Week 6 MCP labs work. Plus, WSL is actually a great way to build software on your Windows PC. You can also skip this final cell, but you will need to come back to this when we start Week 6.<br/>\n",
    "            The WSL Setup instructions are in the Setup folder, <a href=\"../setup/SETUP-WSL.md\">in the file called SETUP-WSL.md here</a>. I do hope this only holds you up briefly - you should be back up and running quickly. Oh the joys of working with bleeding-edge technology!<br/><br/>\n",
    "            With many thanks to student Kaushik R. for raising that this is needed here as well as week 6. Thanks Kaushik!\n",
    "            </span>\n",
    "        </td>\n",
    "    </tr>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from autogen_agentchat.agents import AssistantAgent\n",
    "from autogen_ext.models.openai import OpenAIChatCompletionClient\n",
    "from autogen_ext.tools.mcp import StdioServerParams, mcp_server_tools\n",
    "\n",
    "# Get the fetch tool from mcp-server-fetch.\n",
    "fetch_mcp_server = StdioServerParams(command=\"uvx\", args=[\"mcp-server-fetch\"])\n",
    "fetcher = await mcp_server_tools(fetch_mcp_server)\n",
    "\n",
    "# Create an agent that can use the fetch tool.\n",
    "model_client = OpenAIChatCompletionClient(model=\"gpt-4o-mini\")\n",
    "agent = AssistantAgent(name=\"fetcher\", model_client=model_client, tools=fetcher, reflect_on_tool_use=True)  # type: ignore\n",
    "\n",
    "# Let the agent fetch the content of a URL and summarize it.\n",
    "result = await agent.run(task=\"Review edwarddonner.com and summarize what you learn. Reply in Markdown.\")\n",
    "display(Markdown(result.messages[-1].content))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
